# AGL_Logfile_Parser.py by Tecumseh Fitch and Gesche Westphal Fitch
# Version 0.71 Nov 8 by TF
# Parses a CogLab log file into a list of session information dictionaries (sessionInfo)
# SessionInfo: a dictionary holding all the information for the current session
# If there are errors in the log file, it just writes an error message and omits entire session from sessionList and output file
# Version 0.71 - Near final version;  Seems to be fine, needs some testing
# This is designed to analyze log filess from the forced-choice experiment type used for SOMACCA avian AGL experiments
#Version .73, 28.Jan 2011: updated to parse logfiles containing unrewarded probe trials.
#.76: 29.Jan2011: 40 trials spread over multiple sessions are still written out together as one session (session was  omitted in previous versions).
#80: renamed sessionInfoNames.
#81: prints error meassage and line if a trigger was erroneously shown (Coglab bug, nothing we can do about this at the moment)
#82:writes out total nr of files per session ("session_length",opens whole directory of files to be analysed and saves the
##analysis files in to a separate folder.

sessionInfoNames = ['participant', 'condition', 'testOrTrain', 'line_nr', 'session_nr',
                'session_length', 'correct', 'incorrect', 'correct_train', 'correct_test',"correct_probes", "incorrect_probes","total_probes",
                'correct_train_3n', 'correct_train_2n','correct_test_3n','correct_test_2n', 
                'correct_train_large', 'correct_train_small','correct_test_large','correct_test_small','total_errors','session_length', "temporary_error_count",]
#################################
#Description of sessioInfoNames: The first five names in the list are not reset with every session. i.e. don't move the first five around .
#participant: the individual in the experiment, typically a code, not a name. Derived from the filename
#condition: description of the experimental condtion, eg. (AB)n. Derived from the filename
#test_or_train: whether the session consists of only training sessions or whether it contains test trials. This is derived from the filename.
#session_nr: each full session (i.e. containing max. nr of trials) ,gets a number, starting from 1.
#line_nr: every line in a logfile gets a running number - helpful when finding errors
#session_length: number of trials counted in a session. Is checked against desiredSessionLength to see whether the session is complete.
#total_probes: total number of probe trials found in a session. Is zero if there are no probes.
#correct: nr of times the correct image was chosen first
#incorrect: nr of times the wrong image was chosen first. Correct and incorrect should add up to session_length.
#total_errors: nr of incorrect choices overall within a session, including first incorrect choices
#correct_train: nr of times the correct  image in a training trial was chosen first
#correct_test:nr of times the correct  image in a test trial was chosen first. Is zero if there are no test trials
#correct_train_3n:total nr of times the correct  image (of length 3N)  was chosen first in a training trial (within one full session)
#correct_train_2n:total nr of times the correct  image (of length 2N)  was chosen first in a training trial (within one full session)
#correct_test_3n: total nr of times the correct  image (of length 3N)  was chosen first in a test trial (within one full session)
#correct_test_2n:total nr of times the correct  image (of length 2N)  was chosen first in a test trial (within one full session)
#correct_train_large:total nr of times the correct large image  was chosen first in a training trial (within one full session)
#correct_train_small:total nr of times the correct small image  was chosen first in a training trial (within one full session)
#correct_test_large:total nr of times the correct large image  was chosen first in a test trial (within one full session)
#correct_test:small;total nr of times the correct small image  was chosen first in a test trial (within one full session)
#temporary_error_count: nr of errors are transferred from this counter to total_errors only when a correct choice is made (because a session could be aborted
                                        #before a correct choice is made, and in that case we do not want to count these errors.  Should always be zero in the outfile.
####################################
counterNames = sessionInfoNames[5:] # these are the names of the "counter" entries that should be reset for each new session
def writeHeader(outfile, sessionInfoNames):
    lastIndex = len(sessionInfoNames)-1
    for i, name in enumerate(sessionInfoNames):
        if i < lastIndex:
            outfile.write(str(name) + ",")
        else:
            outfile.write(str(name) + "\n")
            
def writeDictKeys(outfile, theDict, theNames):
    lastIndex = len(theNames)-1
    for i, name in enumerate(theNames):
        if name not in theDict: # just to make sure the name is indeed a key of the dictionary...
            print "\nERROR: WriteDictKeys: key '", name, "' is not in dictionary:"
        if i < lastIndex:
            outfile.write(str(name) + ",")
        else:
            outfile.write(str(name) + "\n")
            
def writeDictValues(outfile, theDict, theNames):
    lastIndex = len(theNames)-1
    for i, name in enumerate(theNames):
        if i < lastIndex:
            outfile.write(str(theDict[name]) + ",")
        else:
            outfile.write(str(theDict[name]) + "\n")        

def initDict(keyNames, initValue = 0):
    # Creates a dictionary containing a key for each item in counterVariableNames
    # Initialize all of the values to initValue (by default, 0)
    return dict.fromkeys(keyNames, initValue)

def resetDict(theDict, whichKeys, initValue = 0):
    # Resets the values of the keys listed in 'whichKeys' to initValue (by default, 0), changing the dict passed in)
    for key in whichKeys:
        if key not in theDict: # make sure the name is indeed a key of the dictionary...
            print "\nERROR: resetDict: key '", key, "' is not in dictionary:"
        theDict[key] = initValue

def initFlagDict():
    # Contains flags for session and trial beginning and ending
    flags = {'awaitingSessionBegin':True, 'awaitingSessionEnd':False, 'awaitingTrialBegin': True, 'awaitingTrialEnd': False,
             'splitSession':False, 'awaitingCorrectChoice': False}
    return flags

def writeSessionList(outfile, sessionList, sessionInfoNames):
    for session in sessionList:
        writeDictValues(outfile, session, sessionInfoNames)

def printSessionList(sessionList, sessionInfoNames):
    print "\n", len(sessionList), "sessions in sessionList"
    for session in sessionList:
        print session

def analyzeSessionList(sessionList):
    anaList = []
    for session in sessionList:
        N = session['session_length']
        numCorrect = session['correct']
        proportionCorrect = numCorrect/float(N)
        anaList.append(proportionCorrect)
    return anaList


############## Below are the CogLab-specific parsing routines: ###########
def parseFileName(infilename):
    outfilename = infilename.split(".")[0]
    chunks_of_file = infilename.split("_")
    participant = chunks_of_file[0]
    condition = chunks_of_file[1]
    subchunk = chunks_of_file[2].split(".")
    test = subchunk[0]
    
    return participant, condition, test, outfilename

def parseTrialBeginLine(line):
    if "TRIGGER" in line:
        print "Trigger shown"
        trial_chunks = line.split()
        pos_stim_chunk = trial_chunks[4]
        image_chunk = pos_stim_chunk.split("[")
        image_with_extras = image_chunk[0]
        image = image_with_extras.split(":")
        print line
        image_only = image[1]
        
    else:
        trial_chunks = line.split()
        pos_stim_chunk = trial_chunks[3]
        image_chunk = pos_stim_chunk.split("[")
        image_with_extras = image_chunk[0]
        image = image_with_extras.split(":")
        image_only = image[1]
        return image_only

def parsePosClickLine(line):
    test_chunks = line.split()
    stim1 = test_chunks[2]
    stim_with_extras = stim1.split(":")
    stim_name = stim_with_extras[1]
    grammar_chunks = stim_name.split("_")
    grammar_length = grammar_chunks[1]
    size_with_extension = grammar_chunks[4]
    size = size_with_extension.split(".")[0]
    return grammar_length, size

def parseLogLine(line, counters, flags):
    def addinterimcounttoErrorcount():
        flags['awaitingCorrectChoice']=False
        counters['incorrect'] += 1
        for _ in range (counters['temporary_error_count']):
            counters['total_errors']+=1
        counters['temporary_error_count'] = 0
        
    # analyse training versus test trials
    # what about undefclicks?
    if "TRIALBEGIN" in line:
        #image_only = parseTrialBeginLine(line) # TF: this is not currently used...
        counters['temporary_error_count'] = 0
        if flags['awaitingTrialBegin'] != True:
            print "ERROR: Trial Begin without Trial End", counters["session_length"]
        else:
            flags['awaitingTrialEnd'] = True
    if 'TRIALEND' in line:
        if flags['awaitingTrialEnd'] != True:
            print "ERROR: TrialEnd without TrialBegin"
        else:
            flags['awaitingTrialEnd'] = False
            flags['awaitingTrialBegin'] = True
        counters['session_length'] +=1
    if "TESTCLICK" in line:
        counters["total_probes"]+=1
        if "STIM1" in line:
            counters["correct_probes"]+=1
        if "STIM2" in line:
            counters["incorrect_probes"]+=1
    if "TRAIN-F	POSCLICK" in line: #this is when the trial has been completed by the bird
        addinterimcounttoErrorcount()
    if "TEST-F	POSCLICK" in line:#this is when the trial has been completed by the bird
        addinterimcounttoErrorcount()       
        
    if "FIRSTPOSCLICK" in line:
        counters['correct'] +=1 # total correct
        flags['awaitingCorrectChoice']=False
        # now analyze test (=test) and training stimuli separately:
        if "TEST-F" in line:
            
            counters['correct_test'] +=1
            grammar_length, size = parsePosClickLine(line)
            if "3" in grammar_length:
                counters['correct_test_3n'] +=1
            if "2" in grammar_length:
                counters['correct_test_2n'] +=1
            if "L" in size:
                counters['correct_test_large'] +=1
            if "S" in size:
                counters['correct_test_small'] +=1
            
        elif "TRAIN-F" in line:
            counters['correct_train'] +=1
            train_grammar_length, train_size = parsePosClickLine(line)
            if "3" in train_grammar_length:
                counters['correct_train_3n'] +=1       
            if "2" in train_grammar_length:    
                counters['correct_train_2n'] +=1
            if "L" in train_size:
                counters['correct_train_large'] +=1
            if "S" in train_size:
                counters['correct_train_small'] +=1
        else:
            print "ERROR: Unknown type of correct answer, linenumber:", counters['line_nr'], line
    else:
        if "NEGCLICK" in line:
            
            counters['temporary_error_count'] +=1
        if 'FIRSTNEGCLICK' in line:
            flags['awaitingCorrectChoice'] = True
            

def parseLogFile(infilename, desiredSessionLength, indirectory):
    # Parse a CogLab logfile into a list of dictionary objects, each dict of the type returned by initDict(sessionInfoNames)
    print "Parsing:", infilename
    sessionInfo = initDict(sessionInfoNames, 0) # initilize the dictionary, using sessionInfoNames as a template
    sessionInfo['participant'], sessionInfo['condition'], sessionInfo['testOrTrain'], _ = parseFileName(infilename)
    sessionInfo['session_nr'] = 1
    sessionInfo['line_nr'] = 0
    flags = initFlagDict()
    sessionList = []
    infile = open(indirectory+"/"+infilename, "rU")
    for line in infile:
        sessionInfo['line_nr'] += 1 # This keeps track of which line number of the logfile we're currently at
        if "NEW SESSION" in line:
            flags['awaitingSessionEnd'] = True
        # Main parsing line: any errors thrown here will generate an error message, but the code keeps going
#        try:
        parseLogLine(line, sessionInfo, flags)
#        except LookupError:
#            print "Bad line at line number: ", sessionInfo['line_nr'], line
#            pass        
        # Check that the session is good, and if session has ended write to the Session List:
        if "SESSION END" in line:
            print line
            if flags['awaitingSessionEnd'] != True:
                print "ERROR: Session End without Session Begin! Session:", sessionInfo['session_nr']
            elif sessionInfo['session_length'] < desiredSessionLength:
                print "ERROR: Session", sessionInfo['session_nr'] , "Ended with", sessionInfo['session_length'], "trials, expecting", desiredSessionLength
                flags['splitSession'] = True
            else:  #  number of trials = desiredSessionLength:
                flags['splitSession'] = False
                flags['awaitingSessionEnd'] = False
                flags['awaitingSessionBegin'] = True
                sessionList.append(dict(sessionInfo)) # if all OK, append a COPY of the dictionary to sessionList
            if flags['splitSession'] == False:
                sessionInfo['session_nr'] +=1#session nr only increases when the full nr of trials is reached
                resetDict(sessionInfo, counterNames, 0) # reinitialize the counters for the next session
            
    infile.close()
    print "Parsing completed."
    return sessionList # contains only "good" sessions, currently.  OK?

def plotSessions(data, ref = 0.5, filename = None):
    from matplotlib import pyplot
    print "Generating Proportion Correct Graph,.."
    pyplot.plot(data, "-")
    pyplot.title("Proportion Correct per Session")
    pyplot.xlabel("Session Number")
    pyplot.ylabel("Proportion Corrent")
    pyplot.plot([0,len(data)], [ref, ref], "--")
    if filename != None:
        pyplot.savefig(filename)
    pyplot.show()
          
############### Beginning of Main Code Block #######################
import os
import Tkinter
from tkFileDialog import askdirectory
top = Tkinter.Tk()
indirectory = askdirectory()
for infilename in os.listdir(indirectory):
    if ".DS" in infilename:
        continue
    sessionlength= 40
    analysis_directory = "AGL_Analysis/"#change the output directory name to  anything you want here
    sessionList = parseLogFile(infilename, sessionlength, indirectory) # The main function of this program.
    # Now write the list out to a file:
    participant, condition, test, outfilename = parseFileName(infilename)
    if not os.path.isdir("./"+ analysis_directory):
        os.mkdir (analysis_directory)
    outfile = open(analysis_directory + outfilename + ".csv", "w")
    writeHeader(outfile, sessionInfoNames)
    writeSessionList(outfile,sessionList, sessionInfoNames)
    outfile.close()
top.mainloop()
##anaList = analyzeSessionList(sessionList)  # do a simple proportion correct analysis
##imageFileName = outfilename + ".svg"
##plotSessions(anaList, 0.5001, imageFileName) # plot it
